package main

import (
    "./parser"
    "./query"
    "config"
    "database/sql"
    "errors"
    "fmt"
    _ "github.com/bmizerany/pq"
    "io/ioutil"
    "log"
    "net/http"
    "path/filepath"
    "time"
    "yotsuba"
)

// Calculate next interval for monitoring loops.
func backoff(refresh time.Duration,
    max time.Duration, rate float64) time.Duration {
    refresh = time.Duration(float64(refresh) * rate)
    if refresh > max {
        refresh = max
    }
    return refresh
}

// Open connection, get posts , close resp.Body immediately. This was moved 
// out of watchThread to isolate http functions and ensure resp.Body is closed
// as soon as possible without leaking.
func getPosts(thread *yotsuba.Thread) (*yotsuba.Thread, int) {
    resp, err := http.Get(thread.Url)
    if err != nil {
        log.Printf("getPosts: error: %s", err.Error())
        return nil, 0
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return nil, resp.StatusCode
    }

    thread, err = parser.ScanThread(thread, &resp.Body)
    if err != nil {
        log.Printf("getPosts: error: %s", err.Error())
        return nil, resp.StatusCode
    }

    return thread, resp.StatusCode
}

// Download image file to the directory given.
func downloadImage(url string, imgNo sql.NullInt64,
    destDir string, insertStmt *sql.Stmt) error {
    number := imgNo.Int64

    resp, err := http.Get(url)
    if err != nil {
        log.Printf("getImage: error retrieving %s; Err: %s", url, err.Error())
        return err
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        respErr := errors.New(fmt.Sprintf("Error retrieving %s, "+
            "received http code %d", url, resp.StatusCode))
        log.Println(respErr)
        return respErr
    }

    imgBytes, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        log.Printf("getImage: %d Couldn't read resp.Body. Err: %s",
            number, err.Error())
        return err
    }

    ext := filepath.Ext(url)
    fileName := fmt.Sprintf("%d%s", number, ext)
    if err = ioutil.WriteFile(destDir+fileName, imgBytes, 0644); err != nil {
        log.Printf("getImage: Unable to create/open image %s; Err: %s",
            fileName, err.Error())
        return err
    }

    // Update images table with path to image.
    if _, err = insertStmt.Exec(fileName, number); err != nil {
        log.Printf("getImage: Could not store file name %s; Err: %s",
            fileName, err.Error())
        return err
    }

    return nil
}

func getImage(url string, imgNo sql.NullInt64, destDir string,
    insertStmt *sql.Stmt, retry int) {

    for i := 0; i < retry; i++ {
        if downloadImage(url, imgNo, destDir, insertStmt) == nil {
            return
        }
        time.Sleep(10 * time.Second)
    }
    log.Printf("Failed to retrieve %s after multiple tries, giving up...", url)
}

// Write post to DB and spawn goroutines to store any related images needed.
func storePost(p *yotsuba.Post, parent int, board *config.Board) error {
    imgName := sql.NullString{}
    imgUrl := sql.NullString{}
    imgNo := sql.NullInt64{}
    bname := board.TableName

    // If post includes image, store image metadata and return row id from
    // image table for reference in post table. Fill out any image related
    // fields that must be stored with the post itself. Note that if the 
    // image data fails to insert (e.g. due to MD5 collision), the crawler
    // will not attempt to retrieve the thumb or the full image itself, and 
    // will bail out of the function instead.
    if p.ThumbUrl != "" {
        row := query.Get(bname, "insertImg").QueryRow(
            p.Img.Md5, p.Img.Size, p.Img.FullX,
            p.Img.FullY, p.Img.ThumbX, p.Img.ThumbY)
        row.Scan(&imgNo)
        if !imgNo.Valid {
            row := query.Get(bname, "getImageByMd5").QueryRow(p.Img.Md5)
            row.Scan(&imgNo)
            if !imgNo.Valid {
                errors.New("Unable to store image nor find dup.")
            }
        }

        if p.ImgName != "" {
            imgName = sql.NullString{p.ImgName, true}
        }
        if p.ImgUrl != "" {
            imgUrl = sql.NullString{p.ImgUrl, true}
        }

        if board.FetchThumbnails {
            go getImage(p.ThumbUrl, imgNo, board.ThumbDir,
                query.Get(bname, "updateImgWithThumbFile"), board.ImgRetry)
        }
        if board.FetchImages {
            go getImage(p.ImgUrl, imgNo, board.ImgDir,
                query.Get(bname, "updateImgWithImgFile"), board.ImgRetry)
        }
    }

    _, err := query.Get(bname, "insertPost").Exec(
        p.Id, nil, p.Time, p.Name, p.Trip, p.Subject, p.Mail, p.PosterUid,
        p.Text, p.Spoiler, parent, imgNo, imgName, imgUrl, nil, nil,
        p.Highlight, yotsuba.AuthType[p.Auth])
    return err
}

// Main thread monitoring loop.
func watchThread(t *yotsuba.Thread, board *config.Board, death chan<- int) {
    refresh := board.ThreadRefreshMin
    knownPosts := map[int]*yotsuba.Post{}
    bname := board.TableName

    var firstPost *yotsuba.Post
    firstRun := true
    for {
        time.Sleep(refresh)

        activity := false
        var status int
        thread, status := getPosts(t)
        if status == http.StatusNotFound {
            log.Printf("%d has 404'd, monitor is quitting\n", t.Id)
            death <- t.Id
            return
        } else if status == http.StatusNotModified ||
            thread == nil ||
            thread.Posts == nil {
            refresh = backoff(refresh, board.ThreadRefreshMax,
                board.ThreadBackoff)
            log.Printf("Thread %d returned status %d, might be dead...",
                t.Id, status)
            continue
        }
        posts := thread.Posts

        firstPost = posts[0]
        // Insert OP and the thread info if this was the first time through.
        if firstRun {
            log.Printf("Inserting thread %d", firstPost.Id)
            _, err := query.Get(bname, "insertThread").Exec(firstPost.Id,
                t.Sticky, t.Locked, false, firstPost.Time, t.Url)
            if err != nil {
                log.Println("DB insertion error: " + err.Error())
            }
            firstRun = false
        }

        // Check for deleted posts and moderator redtext.
        // TODO: Move this into the loop below or investigate set operation
        // libraries to replace these.
        var found bool
        for _, v := range knownPosts {
            found = false
            for _, p := range posts {
                if p.Id == v.Id {
                    if v.FileDeleted != p.FileDeleted {
                        _, err := query.Get(bname, "updatePostBody").Exec(
                            p.Text, v.Id, v.Time)
                        if err != nil {
                            log.Println("DB update error: " + err.Error())
                        }
                    }
                    found = true
                    break
                }
            }

            if !found {
                log.Printf("%d not found in thread %d refresh, "+
                    "marking deleted.", v.Id, t.Id)
                _, err := query.Get(bname, "updateDeletedPost").Exec(
                    v.Id, v.Time)
                if err != nil {
                    log.Println("DB update error: " + err.Error())
                }
                delete(knownPosts, v.Id)
            }
        }

        // Check for new posts.
        for _, p := range posts {
            if knownPosts[p.Id] != nil {
                continue
            }

            if err := storePost(p, firstPost.Id, board); err != nil {
                log.Printf("Post insertion error: %s", err.Error())
            }
            knownPosts[p.Id] = p
            activity = true
        }

        if activity {
            log.Printf("Thread %d was active, resetting refresh rate\n", t.Id)
            refresh = board.ThreadRefreshMin
            continue
        }

        refresh = backoff(refresh, board.ThreadRefreshMax, board.ThreadBackoff)
        log.Printf("Thread %d was inactive, refresh rate changed to %d sec\n",
            t.Id, refresh/time.Second)
    }
}

func getThreads(threads []*yotsuba.Thread, url string,
    page int) []*yotsuba.Thread {
    log.Printf("getThreads: Scanning %s page %d\n", url, page)
    resp, err := http.Get(fmt.Sprintf("%s%d", url, page))
    if err != nil {
        log.Println("getThreads: error: %s. Sleeping.", err.Error())
        return nil
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        log.Printf("getThreads: got status code %d on page %d",
            resp.StatusCode, page)
    }
    for _, t := range parser.ScanPage(&resp.Body, url, page) {
        threads = append(threads, t)
    }
    return threads
}

// watchBoard monitors boards for new threads and spawns new 
// thread monitors as needed.
func watchBoard(board *config.Board) {
    watchedThreads := map[int]*yotsuba.Thread{}
    death := make(chan int, 10) // arbitrary buffer size
    refresh := board.BoardRefreshMin
    bname := board.Abbrev

    log.Printf("WatchBoard: %s\n", bname)
    for {
        activity := false
        foundThreads := []*yotsuba.Thread{}

        // Scan designated pages, collect listing of new threads
        for i := board.StartIndex; i <= board.EndIndex; i++ {
            foundThreads = getThreads(foundThreads, board.Url, i)
        }

        // Monitor last page for ageing threads as well.
        foundThreads = getThreads(foundThreads, board.Url, board.LastPage)

        for _, t := range foundThreads {
            // Spawn thread monitors for any new threads.
            if _, found := watchedThreads[t.Id]; !found {
                activity = true
                go watchThread(t, board, death)
                log.Printf("[%s] spawned a monitor for new thread %d\n",
                    bname, t.Id)
            }
            watchedThreads[t.Id] = t
        }

        if activity {
            refresh = board.BoardRefreshMin
            log.Printf("[%s] new threads found, reset refresh rate\n",
                bname)
        } else {
            refresh = backoff(refresh, board.BoardRefreshMax,
                board.BoardBackoff)
            log.Printf("[%s] No new threads, refresh rate changed to %d sec\n",
                bname, refresh/time.Second)
        }

        log.Printf("[%s] Waiting for dead threads timeout in %d seconds.\n",
            bname, (refresh / time.Second))

        // Sleep until refresh interval passes, but handle any dead threads
        // that come in.
        sleeping := true
        for sleeping {
            sleptAt := time.Now()
            select {
            case dead := <-death:
                t := watchedThreads[dead]
                wasDeleted := t.Page != board.LastPage
                query.Get(board.TableName, "updateExpiredThread").Exec(t.Id, wasDeleted)
                log.Printf("[%s] Thread %d died, removing from "+
                    "monitored threads.\n", bname, dead)
                delete(watchedThreads, dead)
                refresh -= time.Since(sleptAt)
                if refresh < board.BoardRefreshMin {
                    refresh = board.BoardRefreshMin
                }

            case <-time.After(time.Duration(refresh)):
                log.Printf("[%s] Timeout, waking back up "+
                    "and scanning.", bname)
                sleeping = false
            }
        }

    }

}
